/**
 *  JFTSTool - Java Full Text Search tool.
 *  Copyright (C) <2009>  <grupo JFTS>
 *  This program is free software: you can redistribute it and/or modify
 *	it under the terms of the GNU General Public License as published by
 *	the Free Software Foundation, either version 3 of the License, or
 *	(at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *  
 *  You should have received a copy of the GNU General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.jftstool.indexer;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.Date;
import java.util.Map;

import org.apache.log4j.Logger;
import org.apache.log4j.xml.DOMConfigurator;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.jftstool.analyzers.AnalyzerManager;
import org.jftstool.analyzers.exceptions.AnalyzerManagerException;
import org.jftstool.config.JFTSConfiguration;
import org.jftstool.config.SemanticConfig;
import org.jftstool.config.exceptions.JFTSConfigException;
import org.jftstool.exceptions.JFTSFrameworkException;
import org.jftstool.fileParserFramework.FileFormatHandler;
import org.jftstool.fileParserFramework.contentHandler.BodyResultHandler;
import org.jftstool.fileParserFramework.exceptions.FileHandlerException;
import org.jftstool.fileParserFramework.metadata.DocumentMetaData;
import org.jftstool.indexer.document.DBDocument;
import org.jftstool.indexer.document.constructor.DBDocumentsExtractor;
import org.jftstool.indexer.document.constructor.DocumentConstructor;
import org.jftstool.indexer.document.constructor.IDocumentConstructor;
import org.jftstool.indexer.document.exceptions.DBDocumentExtractorException;
import org.jftstool.indexer.document.exceptions.DocumentConstructorException;
import org.jftstool.indexer.document.exceptions.DocumentFormaterException;
import org.jftstool.indexer.document.formater.IDocumentFormater;
import org.jftstool.indexer.exceptions.JFTSIndexerException;
import org.jftstool.semantic.ISemanticModel;
import org.jftstool.semantic.exceptions.JFTSSemanticException;
import org.jftstool.semantic.inference.ISemanticClassInference;
import org.xml.sax.ContentHandler;

/**
 * Class to control the index process
 * @author grupo JTS
 */
public class JFTSIndexer implements IJFTSIndexer {
	private JFTSConfiguration config = null;
	static Logger logger = Logger.getLogger(JFTSIndexer.class);   	
	private Map<String, IndexWriter> iws = null;
	private IndexReader ir = null;
	private ISemanticModel sm = null;
	private boolean useSemanticIndex = false;
	
	/**
	 * Default Constructor
	 * @throws JFTSIndexerException
	 */
	public JFTSIndexer(Map<String, IndexWriter> iws, ISemanticModel sm) throws JFTSIndexerException {
		try {
			config = JFTSConfiguration.getInstance();
			DOMConfigurator.configure("resources/log4j/log4jConfig.xml");
			this.iws = iws;
			this.sm = sm;
			this.useSemanticIndex = config.getIndexerConfig().getUseSemanticOnIndex();
		}
		catch (JFTSConfigException e) {
			throw new JFTSIndexerException(e.getMessage());
		}
	}

	
	/**
	 * Set the map of IndexWriters to use during the index process.
	 * @param iws
	 */
	public void setIndexWriters(Map<String, IndexWriter> iws){
		this.iws = iws;
	}
	
	/**
	 * Set the IndexReader to use during the index process.
	 * @param ir
	 */
	public void setIndexReader(IndexReader ir){
		this.ir = ir;
	}
	
	
	@Override
	public long deleteDocument(String field, String id) throws JFTSIndexerException {
		int deletedDocuments = 0;
		try {
			logger.info("***JFTSTool - Starting delete process***");
			logger.info("Delete pattern [field, value]: [" + field + ", " + id + "]");			
			
			Term delete = new Term(field, id);
			deletedDocuments = ir.deleteDocuments(delete);
			
			logger.info("***JFTSTool - Finish delete process***");			
			logger.info("Documents deleted: " + deletedDocuments);			
			logger.info("Be updated the index when close the JFTS session");
		}
		catch (IOException e) {
			throw new JFTSIndexerException("Error during delete process");			
		}
		return deletedDocuments;
	}	
	
	@Override
	public void addDocument(File file) throws JFTSIndexerException {
		this.addDocument(file, true);
	}

	
	@Override
	public void addDocument(File file, boolean optimizeIndex) throws JFTSIndexerException {
		try {
			logger.info("***JFTSTool - Starting index process***");
			
			//Parse file
			DocumentMetaData metaData = new DocumentMetaData();
			BodyResultHandler result = new BodyResultHandler();			
			_parseDocument(file, result, metaData);
		
			//if parser framework can parse file
			if (result != null){
			
				logger.info("Adding file: " + file.toURI());
				
				//Index file
				_indexFile(result, metaData, optimizeIndex);
				
				logger.info("***JFTSTool - Finish index process***");				
			}
			else{
				logger.info("***JFTSTool - abort index process***");
				throw new JFTSIndexerException("Error to parse file: " + file.getAbsolutePath());				
			}
		}
		catch (FileHandlerException e) {
			logger.info("***JFTSTool - abort index process***");			
			throw new JFTSIndexerException(e.getMessage());
		}
	}

	@Override
	public void addDocuments(File directory, boolean ignoreErrors) throws JFTSIndexerException {
		logger.info("***JFTSTool - Starting index process***");
		
		if (ignoreErrors){
			logger.info("***ignore non critial errors***");
		}
		
		if (!directory.isDirectory()){
			logger.error("FILE isn't a valid direcotry (" + directory.getAbsolutePath() + ")");
			throw new JFTSIndexerException("Error, the file isn't a valid directory. Directory" + directory.getAbsolutePath());			
		}
		else{
			if (!directory.canRead()){
				logger.error("Error, can't read direcotry (" + directory.getAbsolutePath() + ")");				
				throw new JFTSIndexerException("Error, can't read directory " + directory.getAbsolutePath());	
			}
			else{
				Date start = null;
				Date end = null;
				try {
					start = new Date();					
					IDocumentFormater documentFormater = null;
					IDocumentConstructor documentConstructor = null;
					FileFormatHandler ffHandler = new FileFormatHandler();
		
					//get the default document formater
					documentFormater = _getDocumentFormater();
					documentConstructor = new DocumentConstructor();
					
					ISemanticClassInference semClassInf = null;
					if (useSemanticIndex){
						semClassInf = _getClassInference();
					}

					int[] errorCount = new int[]{0};
					
					//Index directory
					_indexDocs(directory, ffHandler, documentFormater, documentConstructor, semClassInf, ignoreErrors, errorCount);
					
					end = new Date();
					
					logger.info("***JFTSTool - Finish index process***");			
					if (ignoreErrors){
						logger.info("Ignore non critial errors: " + errorCount[0]);
					}
					logger.info("Elapsed time: " + (end.getTime() - start.getTime()) + " mseg");						
					
				}
				catch (Throwable e){
					e.printStackTrace();
					end = new Date();					
					logger.info("***JFTSTool - abort index process***");
					System.out.println("Elapsed time: " + (end.getTime() - start.getTime()) + " mseg" );					
					throw new JFTSIndexerException(e.getMessage());					
				}
			}
		}
	}
	
	private void _indexDocs(File file, FileFormatHandler ffHandler, IDocumentFormater documentFormater,
					IDocumentConstructor documentConstructor, ISemanticClassInference semClassInf,
					boolean ignoreErrors, int[] errorCount) throws JFTSIndexerException {
		//if file is a directory
	    if (file.isDirectory()){
	    	//if can read directory
	    	if (file.canRead()){
	    		String[] files = file.list();
	    		
	    		if (files != null){
		    		Arrays.sort(files);			  
		    		for (int i = 0; i < files.length; i++){
		    			_indexDocs(new File(file, files[i]), ffHandler, documentFormater, documentConstructor, semClassInf, ignoreErrors, errorCount);
	    			}
	    		}
	    		else{
	    			if (!ignoreErrors){
		    			throw new JFTSIndexerException("Error, can't get files from: " + file.getAbsolutePath());
		    		}
		    		else{
		    			logger.warn("WARNING, can't get files from: " + file.toURI());
		    			errorCount[0]++;		    			
		    		}	    			
	    		}
	    	}
	    	else{
	    		if (!ignoreErrors){
	    			throw new JFTSIndexerException("Error, can't read file: " + file.getAbsolutePath());
	    		}
	    		else{
	    			logger.warn("WARNING, can't read file: " + file.toURI());
	    			errorCount[0]++;	    			
	    		}
	    	}
	    }
	    else {
			DocumentMetaData metaData = new DocumentMetaData();	    	
			BodyResultHandler resultHandler = new BodyResultHandler();
			String language = null;			
			IndexWriter iw = null;
						
	    	try {
	    		logger.info("Analizing file: " + file.toURI());	    		
	    		ffHandler.getDocument(file, resultHandler, metaData);
				
	    		//if parser framework can parse file
				if (resultHandler != null){
					
					if (!config.getInitConfig().getMultiLanguageIndex()){
						//Get default system language
						language = config.getDefaultLanguage();
						if (language == null){
							language = config.getLanguageDetectorConfig().getUnknownLanguage();
						}
					}
					else{
						language = metaData.getProperty(DocumentMetaData.CONTENT_LANGUAGE);
					}

					//Get the indexWriter for languge
					iw = _getIndexWriter(language);
					
					if (iw == null){
						if (!ignoreErrors){						
							throw new JFTSIndexerException("Error to get indeWriter for language: " + language);
						}
						else{
							logger.warn("WARNING, can't find indexWriter for document language: " + language + " file :" + file.toURI());
						}	
					}
					else{
						logger.info("Indexing file: " + file.toURI());
					
						//Build dinamic Lucene Document to index
						Document d = _buildJFTSDocument(documentConstructor, documentFormater, semClassInf, null, resultHandler, metaData);						
	
						//Index document
						iw.addDocument(d);
					}
				}
				else{
					if (!ignoreErrors){
						throw new JFTSIndexerException("Error to parse file: " + file.getAbsolutePath());
					}
					else{
						logger.warn("WARNING, can't parse file: " + file.toURI());
		    			errorCount[0]++;						
					}
				}				
			}
	    	catch (Exception e) {
	    		if (!ignoreErrors){
	    			e.printStackTrace();
	    			throw new JFTSIndexerException(e.getMessage());
	    		}
	    		else{
	    			logger.warn("WARNING, an error occurred during the indexing process");
	    			errorCount[0]++;	    			
	    		}
			}
	    }
	 }		
	
		
	private void _parseDocument(File file, ContentHandler resultHandler, DocumentMetaData metaData) throws FileHandlerException {
		FileFormatHandler handler = new FileFormatHandler();
		handler.getDocument(file, resultHandler, metaData);	
	}
	
	
	
	
	/**
	 * Add document to the JFTS index 
	 * @param text Texto plano obtenido luego de parsear el documento
	 * @param metaData MetaData extraida del documento
	 * @param optimizeIndex Optimiza o no el indice luego de indexar el documento
	 * @throws JFTSIndexerException
	 */
	private void _indexFile(ContentHandler text, DocumentMetaData metaData, boolean optimizeIndex) throws JFTSIndexerException {
		String indexPath = null;
		IndexWriter iw = null;
		IDocumentFormater documentFormater = null;
		String language = null;
		try {
			
			//Get document language
			language = metaData.getProperty(DocumentMetaData.CONTENT_LANGUAGE);
			
			iw = _getIndexWriter(language);
			if (iw != null){
				
				//Build dinamic Lucene Document to index
				documentFormater = _getDocumentFormater();
				DocumentConstructor documentConstructor = new DocumentConstructor();
				
				ISemanticClassInference semClassInf = null;
				if (useSemanticIndex){
					semClassInf = _getClassInference();
				}
				
				Document d = _buildJFTSDocument(documentConstructor, documentFormater, semClassInf, null, text, metaData);				
			
				iw.addDocument(d);
				
				//If param optimizeIndex is equals true, proceed to optimize
				if (optimizeIndex){
					logger.info("optimize index");			
					iw.optimize();
				}
			}
			else{
				throw new JFTSIndexerException("Error to get indeWriter for language: " + language);					
			}
		}
		catch (IOException e) {
			throw new JFTSIndexerException("Error to open index. Index dir: " + indexPath);
		}
		catch (DocumentConstructorException e) {
			throw new JFTSIndexerException(e.getMessage());
		}
		catch (DocumentFormaterException e) {
			throw new JFTSIndexerException(e.getMessage());			
		}
		catch (JFTSSemanticException e) {
			throw new JFTSIndexerException(e.getMessage());			
		}
	}
	
	private IDocumentFormater _getDocumentFormater() throws JFTSIndexerException {
		IDocumentFormater documentFormater = null;
		String className = null;
		try {
			className = config.getDocumentConfig().getDocumentFormaterClass();
			Class<? extends IDocumentFormater> docFormClass;
			docFormClass = Class.forName(className).asSubclass(IDocumentFormater.class);
			documentFormater = (IDocumentFormater) docFormClass.newInstance();			
		}
		catch (JFTSConfigException e) {
			throw new JFTSIndexerException(e.getMessage());			
		}
		catch (ClassNotFoundException e) {
			throw new JFTSIndexerException("Error, cant't find class: " + className);
		}
		catch (InstantiationException e) {
			throw new JFTSIndexerException("Error to load class: " + className);
		}
		catch (IllegalAccessException e) {
			throw new JFTSIndexerException("Error to load class: " + className);			
		}
		return documentFormater;
	}
	
	private ISemanticClassInference _getClassInference() throws JFTSIndexerException {
		ISemanticClassInference classInferece = null;
		String className = null;
		try {
			className = config.getSemanticConfig().getClassInferenceClass();
			Class<? extends ISemanticClassInference> infClass;
			infClass = Class.forName(className).asSubclass(ISemanticClassInference.class);
			Constructor<? extends ISemanticClassInference> c = infClass.getConstructor(SemanticConfig.class);			
			classInferece = c.newInstance(config.getSemanticConfig()); 
		}
		catch (JFTSConfigException e) {
			throw new JFTSIndexerException(e.getMessage());			
		}
		catch (ClassNotFoundException e) {
			throw new JFTSIndexerException("Error, cant't find class: " + className);
		}
		catch (InstantiationException e) {
			throw new JFTSIndexerException("Error to load class: " + className);
		}
		catch (IllegalAccessException e) {
			throw new JFTSIndexerException("Error to load class: " + className);			
		}
		catch (SecurityException e) {
			throw new JFTSIndexerException("Error, cant't load class: " + className);
		}
		catch (NoSuchMethodException e) {
			throw new JFTSIndexerException("Error, cant't find constructor for class: " + className);
		}
		catch (IllegalArgumentException e) {
			throw new JFTSIndexerException("Error, cant't load class: " + className);			
		}
		catch (InvocationTargetException e) {
			throw new JFTSIndexerException("Error, cant't load class: " + className);			
		}
		return classInferece;
	}	

	
	@Override
	public void addDBDocuments(String useConnection, String dbQuery, boolean ignoreErrors) throws JFTSIndexerException {
		logger.info("***JFTSTool - Starting DataBase index process***");		

		if (ignoreErrors){
			logger.info("***ignore non critial errors***");
		}
	
		Date start = null;
		Date end = null;

		try {		
			start = new Date();					
			IDocumentFormater documentFormater = null;
			IDocumentConstructor documentConstructor = null;
			FileFormatHandler ffHandler = new FileFormatHandler();
	
			//Get default system language
			String defaultLanguage = config.getDefaultLanguage();
			if (defaultLanguage == null){
				defaultLanguage = config.getLanguageDetectorConfig().getUnknownLanguage();
			}
	
			//get the default document formater
			documentFormater = _getDocumentFormater();
			documentConstructor = new DocumentConstructor();
			ISemanticClassInference semClassInf = null;
			if (useSemanticIndex){
				semClassInf = _getClassInference();
			}			

			int[] errorCount = new int[]{0};
	
			//Index DataBase documents
			_indexDBDocs(useConnection, dbQuery, ffHandler, documentFormater, documentConstructor, semClassInf, ignoreErrors, errorCount);
			
			end = new Date();
	
			logger.info("***JFTSTool - Finish index process***");			
			if (ignoreErrors){
				logger.info("Ignore non critial errors: " + errorCount[0]);
			}
			logger.info("Elapsed time: " + (end.getTime() - start.getTime()) + " mseg");			
		}
		catch (Throwable e) {
			end = new Date();					
			logger.info("***JFTSTool - abort DataBase index process***");
			System.out.println("Elapsed time: " + (end.getTime() - start.getTime()) + " mseg" );
			e.printStackTrace();
			throw new JFTSIndexerException(e.getMessage());					
		}
	}
	
	private void _indexDBDocs(String dbConn, String dbQuery, FileFormatHandler ffHandler,
					IDocumentFormater documentFormater, IDocumentConstructor documentConstructor,
					ISemanticClassInference semClassInf, boolean ignoreErrors, int[] errorCount) 
					throws JFTSIndexerException {
		
		DBDocumentsExtractor extractor = null;
		int count = 1;
		try{
			extractor = _getDocumentExtractor();
			extractor.createConnection(dbConn);
			extractor.createStatement(dbQuery);			
			
			DBDocument doc = null;
			DocumentMetaData metaData = null;	    	
			BodyResultHandler resultHandler = null;
			String language = null;
			IndexWriter iw = null;
			
			while ((doc = extractor.next()) != null){
				try{
					if (doc.getDocument() != null){
						metaData = new DocumentMetaData();
						resultHandler = new BodyResultHandler();
						ffHandler.getDocument(new ByteArrayInputStream(doc.getDocument()), resultHandler, metaData);
						if (resultHandler != null){
							
							if (!config.getInitConfig().getMultiLanguageIndex()){
								//Get default system language
								language = config.getDefaultLanguage();
								if (language == null){
									language = config.getLanguageDetectorConfig().getUnknownLanguage();
								}
							}
							else{
								language = metaData.getProperty(DocumentMetaData.CONTENT_LANGUAGE);
							}
							//Get the indexWriter for languge
							iw = _getIndexWriter(language);
							
							if (iw == null){
								if (!ignoreErrors){						
									throw new JFTSIndexerException("Error to get indeWriter for language: " + language);
								}
								else{
									logger.warn("WARNING, can't find indexWriter for document language: " + language);
								}	
							}
							else{
								//Build dinamic Lucene Document to index
								Document d = _buildJFTSDocument(documentConstructor, documentFormater, semClassInf, doc, resultHandler, metaData);
								
								logger.info("Adding dbDocument count: " + count++);							
								iw.addDocument(d);
							}
						}
						else{
							if (!ignoreErrors){
								throw new JFTSIndexerException("Error to parse db document");
							}
							else{
								logger.warn("WARNING, Error to parse db document");
								errorCount[0]++;
							}
						}				
					}
				}
				catch (CorruptIndexException e) {
					throw new JFTSIndexerException(e.getMessage());
				}
				catch (IOException e) {
					throw new JFTSIndexerException(e.getMessage());
				}
				catch (Exception e){
					if (!ignoreErrors){
						e.printStackTrace();
						throw new JFTSIndexerException(e.getMessage());
					}
					else{
						logger.warn("WARNING, an error occurred during the indexing process: " + e.getMessage());
						errorCount[0]++;
					}
				}
			}
		}
		catch(DBDocumentExtractorException e){
			throw new JFTSIndexerException(e.getMessage());			
		}
	}
	
	private DBDocumentsExtractor _getDocumentExtractor() throws JFTSIndexerException {
		DBDocumentsExtractor documentExtractor = null;
		String className = null;
		try {
			className = config.getDBConfig().getClassExtractor();
			System.out.println("DocumentExtractor: "+ className);
			Class<? extends DBDocumentsExtractor> docExtClass;
			docExtClass = Class.forName(className).asSubclass(DBDocumentsExtractor.class);
			documentExtractor = (DBDocumentsExtractor) docExtClass.newInstance();			
		}
		catch (JFTSConfigException e) {
			throw new JFTSIndexerException(e.getMessage());			
		}
		catch (ClassNotFoundException e) {
			throw new JFTSIndexerException("Error, cant't find class: " + className);
		}
		catch (InstantiationException e) {
			throw new JFTSIndexerException("Error to load class: " + className);
		}
		catch (IllegalAccessException e) {
			throw new JFTSIndexerException("Error to load class: " + className);			
		}
		return documentExtractor;
	}
	
	
	/**
	 * <p>Returns the language indexWriter received for the parameter<br>
	 * if there indexWriter to the language, if JFTS is configured to<br>
	 * use the default indexWriter returns this indexWriter.</p>	 
	 * @param language Lenguaje del indexWriter a obtener
	 * @return
	 * @throws JFTSFrameworkException
	 */
	private IndexWriter _getIndexWriter(String language) throws JFTSIndexerException{
		IndexWriter iw = null;
		try {
			String unsupportedLanguage = config.getLanguageDetectorConfig().getUnknownLanguage();
			AnalyzerManager mgr = new AnalyzerManager();
			if (language.trim().equalsIgnoreCase(unsupportedLanguage.trim())){
				if (config.getIndexerConfig().getUseDefaultAnalyzer()){
					language = mgr.getDefaultLanguage();
				}
			}
			iw = iws.get(language);
			
			if (iw == null){
				if (config.getIndexerConfig().getUseDefaultIndexer()){
					language = mgr.getDefaultLanguage();
				}
				iw = iws.get(language);
			}
		}
		catch (JFTSConfigException e) {
			throw new JFTSIndexerException(e.getMessage());			
		}
		catch (AnalyzerManagerException e) {
			throw new JFTSIndexerException(e.getMessage());			
		}		
		return iw;
	}

	private Document _buildJFTSDocument(IDocumentConstructor docConstructor, IDocumentFormater docFormater,
			ISemanticClassInference semClassInf, DBDocument dbDoc, ContentHandler text, DocumentMetaData metaData)
			throws DocumentFormaterException, DocumentConstructorException, JFTSSemanticException {

		if (useSemanticIndex){
			semClassInf.anotateDocument(sm, dbDoc, text, metaData);
		}		
		
		org.jdom.Document formatDoc = (dbDoc == null) ? docFormater.formatDocument(text, metaData) : docFormater.formatDocument(dbDoc, text, metaData);
		
		Document d = docConstructor.buildJFTSDocument(formatDoc);
	
		return d;
	}
}